#!/usr/bin/python
#coding=utf-8
'''
    4.13 创建数据处理管道
    想以管道的方式 处理数据。应对不能一次性放入内存的情况
'''

# 生成器函数是一个实现管道机制的好办法

# 定义一个由多个执行特定任务独立任务的 简单生成器函数 组成的容器
import os
import fnmatch
import gzip
import bz2
import re

def gen_find(filepat, top):
    '''
    Find all filenames in a directory tree that
    match a shell wildcard pattern
    :param filepat:
    :param top:
    :return:
    '''
    for path, dirlist, filelist in os.walk(top):
        for name in fnmatch.filter(filelist, filepat):
            yield os.path.join(path, name)

def gen_opener(filenames):
    ''''''
    for filename in filenames:
        if filename.endswith('.gz'):
            f = gzip.open(filename, 'rt')
        elif filename.endswith('.bz2'):
            f = bz2.open(filename, 'rt')
        else:
            f = open(filename, 'rt')
        yield f
        f.close()

def gen_concatenate(iterators):
    '''
    Chain a sequence of iterators together into a single sequence.
    :param iterators:
    :return:
    '''
    for it in iterators:
        yield from it

def gen_grep(pattern, lines):
    '''
    Look for a regex pattern in a sequence of lines
    :param pattern:
    :param lines:
    :return:
    '''
    pat = re.compile(pattern)
    for line in lines:
        if pat.search(line):
            yield line

# 现在可以将这些函数连接起来 创建一个 处理管道。
# 例如：查找包含单词python的所有日志行
logname = gen_find('access-log*', 'www')
files = gen_opener(logname)
lines = gen_concatenate(files)
pylines = gen_grep('(?i)python', lines)
for line in pylines:
    print(line)

# 扩展管道 --> 在生成器表达式中包装数据
# 例如，计算出传输的字节数并计算其综合
lognames = gen_find('access-log*', 'www')
files = gen_opener(logname)
lines = gen_concatenate(files)
pylines = gen_grep('(?i)python', lines)
bytecolumn = (line.rsplit(None, 1)[1] for line in pylines)
bytes = (int(x) for x in bytecolumn if x != '-')
print('Total', sum(bytes))

# yield语句作为数据的生产者
# 生成器管道